from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

print "program begins..."

queryDict = {}

# example: 
# key: length of the query
# value: how many queries have that length
queryLengthOfLengthDict = {}

inputQueryFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/tail5KFrom100KQueries"
inputQueryHandler = open(inputQueryFileName,"r")

for line in inputQueryHandler.readlines():
    elements = line.strip().split(":")
    queryID = int(elements[0])
    
    data = elements[1]
    data = data.lower()
    
    for i in range(0,len(data)):
        # print "data[i]:",ord(data[i])
        if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
            # Just replace them with a space.
            data = data[:i] + " " + data[i+1:]
    
    queryContent = data
    
    # compute the length of the current query
    lengthOfCurrentQuery = len( queryContent.strip().split(" ") )
    if lengthOfCurrentQuery not in queryLengthOfLengthDict:
        queryLengthOfLengthDict[lengthOfCurrentQuery] = 1
    else:
        queryLengthOfLengthDict[lengthOfCurrentQuery] += 1
    
    if queryID not in queryDict:
        queryDict[queryID] = queryContent

print "----->","len(queryDict):",len(queryDict)
print "----->","len(queryLengthOfLengthDict):",len(queryLengthOfLengthDict)
print "----->","queryLengthOfLengthDict:",queryLengthOfLengthDict

totalNumOfQueries = 0
totalLengthAmongAllQueries = 0
for key in queryLengthOfLengthDict:
    totalNumOfQueries += queryLengthOfLengthDict[key]
    totalLengthAmongAllQueries += queryLengthOfLengthDict[key] * key

print "totalNumOfQueries:",totalNumOfQueries
print "totalLengthAmongAllQueries:",totalLengthAmongAllQueries
print "averageLengthOfQuery:",totalLengthAmongAllQueries/5000
print "random guess performance:"
# Let's continue to compute the expected random guess baseline
indexKeptPercentageList = [0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.05, 0.01]
for key1 in indexKeptPercentageList:
    currentIndexKeptPR1 = 0.0
    currentIndexKeptPR2 = 0.0
    for key2 in queryLengthOfLengthDict:
        s_i = queryLengthOfLengthDict[key2] / 5000
        currentIndexKeptPR1 += s_i * pow(key1,key2)
        currentIndexKeptPR2 += s_i * pow(key1,s_i)
    # print key1,"(    right):",currentIndexKeptPR1
    # print currentIndexKeptPR1
    
    # Just not outputing this
    # print key1,"(not right):",currentIndexKeptPR2
    print currentIndexKeptPR2

print "program ends."












